{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import cPickle\n",
    "import itertools\n",
    "import datetime\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy import sparse\n",
    "from collections import defaultdict\n",
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import normalize\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import hashlib\n",
    "\n",
    "def HashEncoding(num):\n",
    "  if len(num.strip()) == 0:\n",
    "    return -1\n",
    "  else:\n",
    "    return int(hashlib.sha224(num).hexdigest()[0:4], 16)\n",
    "\n",
    "def RemoveNaNValue(num):\n",
    "  if len(num.strip()) == 0:\n",
    "    return 0.0\n",
    "  else:\n",
    "    return float(num)\n",
    "\n",
    "def StartTime(date):\n",
    "  if date == 'NaN':\n",
    "    return 0.0\n",
    "  else:\n",
    "    st = datetime.datetime.strptime(date, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "    return (st.year)*12 + st.month    \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_csv = pd.read_csv(\"test.csv\")\n",
    "train_csv = pd.read_csv(\"train.csv\")\n",
    "del train_csv['interested']\n",
    "del train_csv['not_interested']\n",
    "\n",
    "train_test_csv = pd.concat([train_csv,test_csv])\n",
    "\n",
    "UniqueEvents_data = train_test_csv.drop_duplicates(['event'])\n",
    "Number_Of_UniEvents = len(UniqueEvents_data)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "################################################参考答案的代码    \n",
    "#统计训练集中有多少不同的用户和不同的events\n",
    "uniqueEvents = set()\n",
    "for filename in [\"train.csv\", \"test.csv\"]:\n",
    "    f = open(filename, 'rb')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    f.readline().strip().split(\",\")\n",
    "    \n",
    "    for line in f:    #对每条记录\n",
    "        columns = line.strip().split(\",\")\n",
    "        uniqueEvents.add(columns[1])   #第二列为活动ID\n",
    "f.close()\n",
    "\n",
    "#重新编码活动索引字典   \n",
    "eventIndex = dict()\n",
    "for i, e in enumerate(uniqueEvents):\n",
    "    eventIndex[e] = i\n",
    "\n",
    "\n",
    "EventMatrix = ss.dok_matrix((Number_Of_UniEvents, 109))\n",
    "\n",
    "fin = open(\"events.csv\", 'rb')\n",
    "fin.readline() \n",
    "for line in fin.readlines():\n",
    "    cols = line.strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    \n",
    "    if eventIndex.has_key(eventId):  \n",
    "        i = eventIndex[eventId]\n",
    "        EventMatrix[i, 0] = StartTime(cols[2])\n",
    "        for a in range(1,4):            \n",
    "            EventMatrix[i, a] = HashEncoding(cols[a+2])\n",
    "        for b in range(5,6):\n",
    "            EventMatrix[i, b] = RemoveNaNValue(cols[b+2]) \n",
    "        for c in range(9, 109):\n",
    "            EventMatrix[i, c-2] = cols[c]\n",
    "fin.close()\n",
    "\n",
    "EventMatrix = normalize(EventMatrix, norm=\"l2\",copy=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  9.99999990e-01  -4.14010098e-05  -4.14010098e-05 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]\n",
      " [  3.20791657e-01   7.68988893e-01   5.52945261e-01 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]\n",
      " [  6.43767752e-01   6.68288176e-01   3.72763757e-01 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]\n",
      " ..., \n",
      " [  9.99999977e-01  -4.13992953e-05  -4.13992953e-05 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]\n",
      " [  9.99999903e-01  -4.13975784e-05  -4.13975784e-05 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]\n",
      " [  5.81032963e-01   8.13879126e-01  -2.40543557e-05 ...,   0.00000000e+00\n",
      "    0.00000000e+00   0.00000000e+00]]\n"
     ]
    }
   ],
   "source": [
    "X_train_part = EventMatrix \n",
    "print(X_train_part.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 一个参数点（聚类数据为K）的模型，在校验集上评价聚类算法性能\n",
    "# def K_cluster_analysis(K, X_train, y_train, X_val, y_val):\n",
    "def K_cluster_analysis(K, X_train):    \n",
    "    start = time.time()   \n",
    "    print(\"K-means begin with clusters: {}\".format(K));\n",
    "    \n",
    "    #K-means,在训练集上训练\n",
    "    mb_kmeans = MiniBatchKMeans(n_clusters = K)\n",
    "    mb_kmeans.fit(X_train)\n",
    "    \n",
    "    # 在训练集和测试集上测试\n",
    "    #y_train_pred = mb_kmeans.fit_predict(X_train)\n",
    "    y_val_pred = mb_kmeans.predict(X_train)\n",
    "    \n",
    "    #以前两维特征打印训练数据的分类结果\n",
    "    #plt.scatter(X_train[:, 0], X_train[:, 1], c=y_pred)\n",
    "    #plt.show()\n",
    "\n",
    "    # K值的评估标准\n",
    "    #常见的方法有轮廓系数Silhouette Coefficient和Calinski-Harabasz Index\n",
    "    #这两个分数值越大则聚类效果越好\n",
    "    #CH_score = metrics.calinski_harabaz_score(X_train,mb_kmeans.predict(X_train))\n",
    "    CH_score = metrics.silhouette_score(X_train, y_val_pred)\n",
    "    \n",
    "#     #也可以在校验集上评估K\n",
    "#     v_score = metrics.v_measure_score(y_val, y_val_pred)\n",
    "    \n",
    "    end = time.time()\n",
    "    print(\"CH_score: {}, time elaps:{}\".format(CH_score, int(end-start)))\n",
    "#     print(\"v_score: {}\".format(v_score))\n",
    "    \n",
    "#     return CH_score,v_score\n",
    "    return CH_score\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-means begin with clusters: 10\n",
      "CH_score: 0.79010969545, time elaps:28\n",
      "K-means begin with clusters: 20\n",
      "CH_score: 0.73765867796, time elaps:28\n",
      "K-means begin with clusters: 30\n",
      "CH_score: 0.301828080309, time elaps:16\n",
      "K-means begin with clusters: 40\n",
      "CH_score: 0.353941788773, time elaps:22\n",
      "K-means begin with clusters: 50\n",
      "CH_score: 0.172539224495, time elaps:24\n",
      "K-means begin with clusters: 60\n",
      "CH_score: 0.196831361523, time elaps:23\n",
      "K-means begin with clusters: 70\n",
      "CH_score: 0.226238939568, time elaps:13\n",
      "K-means begin with clusters: 80\n",
      "CH_score: 0.310414433591, time elaps:16\n",
      "K-means begin with clusters: 90\n",
      "CH_score: 0.246649601266, time elaps:25\n",
      "K-means begin with clusters: 100\n"
     ]
    }
   ],
   "source": [
    "# 设置超参数（聚类数目K）搜索范围\n",
    "Ks = [10,20,30,40,50,60,70,80,90,100]\n",
    "CH_scores = []\n",
    "for K in Ks:\n",
    "    ch = K_cluster_analysis(K, X_train_part)\n",
    "    CH_scores.append(ch)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制不同PCA维数下模型的性能，找到最佳模型／参数（分数最高）\n",
    "plt.plot(Ks, np.array(CH_scores), 'b-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "K-means begin with clusters: 10\n",
    "CH_score: 0.744497772439, time elaps:34\n",
    "K-means begin with clusters: 20\n",
    "CH_score: 0.744883018427, time elaps:17\n",
    "K-means begin with clusters: 30\n",
    "CH_score: 0.593133151492, time elaps:13\n",
    "K-means begin with clusters: 40\n",
    "CH_score: 0.449377092681, time elaps:14\n",
    "K-means begin with clusters: 50\n",
    "CH_score: 0.324431997835, time elaps:14\n",
    "K-means begin with clusters: 60\n",
    "CH_score: 0.321051146228, time elaps:17\n",
    "K-means begin with clusters: 70\n",
    "CH_score: 0.234091252118, time elaps:13\n",
    "K-means begin with clusters: 80\n",
    "CH_score: 0.222469622783, time elaps:15\n",
    "K-means begin with clusters: 90\n",
    "CH_score: 0.271076894367, time elaps:13\n",
    "K-means begin with clusters: 100\n",
    "CH_score: 0.186958834732, time elaps:15\n",
    "\n",
    "\n",
    "k值为20时候， CH-score最高，推测最佳模型k为20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
